# -*- coding:utf-8 -*-
'''
目标网址：https://www.mafengwo.cn/gonglve/
需求: 抓取该网址下推荐攻略
目标字段如下图 没有的字段置空处理或者为None 并获取详情页链接与详情页中
所有内容文字图片可以不管，最好是保存原有的格式。
一共是 来自、蜂蜂体验过、推荐标题、浏览量、评论、详情页url、详情页内容7个字段
'''
import requests
from lxml import etree
import csv
import random


def get_start_data():
    list1 = []

    url = "https://api.hailiangip.com:8522/api/getIpEncrypt?dataType=0&encryptParam=SlDyzgfgDW12vuaMHmQkMz9pKEmWH7kDAoD1ZC4KkxpHXuvLm%2B3L9xWaasGPtq4T4c1NrsDYZz0rKXbmVK5B4QBG7L4Me9DYR7C%2FmLj0ijWqRxTDf5%2F5k7dF8PzzjCTrmjnX%2FnL660JPzNmFbcKWe0Y75OsMrICvGcnqo6oq1zpYDcvC%2FeTFVDIQgCd6Bl9z0AOtabf92lu9Nd5cdhcIj9TXdJz%2BFe7XNmfGghK3pfs%3D"

    dict_data = requests.get(url).json()

    proxy = {
        "https://": str(dict_data["data"][0]["ip"]) + ":" + str(dict_data["data"][0]["port"])

    }
    cookies = {
        'PHPSESSID': 'ndsfh7lp6ochab3rok923u4lg4',
        'mfw_uuid': '653f52bc-914c-8441-6d82-b57066450c1c',
        'oad_n': 'a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222023-10-30+14%3A52%3A44%22%3B%7D',
        '__jsluid_s': '9cf863bb9c9c7a233b17e918f2228c74',
        '__omc_chl': '',
        '__omc_r': '',
        '__mfwc': 'direct',
        '__mfwa': '1698648764837.76974.1.1698648764837.1698648764837',
        '__mfwlv': '1698648764',
        '__mfwvn': '1',
        'uva': 's%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1698648764%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B',
        '__mfwurd': 'a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1698648764%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D',
        '__mfwuuid': '653f52bc-914c-8441-6d82-b57066450c1c',
        'Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0': '1698648765',
        'bottom_ad_status': '0',
        '__jsl_clearance_s': '1698648973.248|0|%2BjVl6bSEUWSUDhit2gEbxaVAIC0%3D',
        '__mfwb': '7f307f7e5900.3.direct',
        '__mfwlt': '1698649042',
        'Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0': '1698649042',
    }

    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        # 'Cookie': 'PHPSESSID=ndsfh7lp6ochab3rok923u4lg4; mfw_uuid=653f52bc-914c-8441-6d82-b57066450c1c; oad_n=a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222023-10-30+14%3A52%3A44%22%3B%7D; __jsluid_s=9cf863bb9c9c7a233b17e918f2228c74; __omc_chl=; __omc_r=; __mfwc=direct; __mfwa=1698648764837.76974.1.1698648764837.1698648764837; __mfwlv=1698648764; __mfwvn=1; uva=s%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1698648764%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B; __mfwurd=a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1698648764%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D; __mfwuuid=653f52bc-914c-8441-6d82-b57066450c1c; Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698648765; bottom_ad_status=0; __jsl_clearance_s=1698648973.248|0|%2BjVl6bSEUWSUDhit2gEbxaVAIC0%3D; __mfwb=7f307f7e5900.3.direct; __mfwlt=1698649042; Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698649042',
        'Pragma': 'no-cache',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46',
        'sec-ch-ua': '"Chromium";v="118", "Microsoft Edge";v="118", "Not=A?Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
    }

    response = requests.get('https://www.mafengwo.cn/gonglve/', cookies=cookies, headers=headers).text
    html = etree.HTML(response)
    data_list = html.xpath("//div[@class='gl-post']//div[@class='_j_feed_data']//div[@class='feed-item _j_feed_item']")
    for num, data in enumerate(data_list):
        href = data.xpath("./a/@href")[0]
        # num=href.split("/")[-1] + ".html"
        num = f"one-{num}"
        from_one = "来自" + data.xpath("./a/div[1]/span[2]/strong/text()")[0]
        title = (data.xpath("./a/div[2]/text()")[0]).strip()
        try:
            fengfeng = data.xpath("./a/div[1]/span[1]/span/text()")[0] + (
            data.xpath("./a/div[1]/span[1]/text()")[0]).strip()
        except:
            fengfeng = "None"
        try:
            liulan = data.xpath("./a/div[4]/ul/li[4]/text()")[0]
        except:
            liulan = "None"
        try:
            pinglun = data.xpath("./a/dl/dd/div[2]/span[2]/text()")[0]
        except:
            pinglun = "None"
        # print(href, from_one, title, fengfeng, liulan, pinglun)
        list1.append(href)
        list1.append(from_one)
        list1.append(title)
        list1.append(fengfeng)
        list1.append(liulan)
        list1.append(pinglun)
        list1.append("\n")
        xiang_data(href, proxy, num)

    # 创建csv文档对象
    with open('document_data.csv', 'w', newline='', encoding="utf-8") as csvfile:
        writer = csv.writer(csvfile)
        # 写入数据
        writer.writerow(list1)


def post_start_data():
    cookies = {
        'mfw_uuid': '653f52bc-914c-8441-6d82-b57066450c1c',
        'oad_n': 'a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222023-10-30+14%3A52%3A44%22%3B%7D',
        '__jsluid_s': '9cf863bb9c9c7a233b17e918f2228c74',
        '__omc_chl': '',
        '__omc_r': '',
        '__mfwc': 'direct',
        'uva': 's%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1698648764%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B',
        '__mfwurd': 'a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1698648764%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D',
        '__mfwuuid': '653f52bc-914c-8441-6d82-b57066450c1c',
        'bottom_ad_status': '0',
        '__mfwa': '1698648764837.76974.3.1698653932344.1698656825781',
        '__mfwlv': '1698656825',
        '__mfwvn': '2',
        'PHPSESSID': 'nl5p6aadgj31qpm1pvckgj4a20',
        'Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0': '1698648765,1698654537,1698656826,1698656945',
        '__mfwb': 'ad354a1a267b.3.direct',
        '__mfwlt': '1698656964',
        'Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0': '1698656965',
    }

    headers = {
        'authority': 'www.mafengwo.cn',
        'accept': '*/*',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'cache-control': 'no-cache',
        'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
        # 'cookie': 'mfw_uuid=653f52bc-914c-8441-6d82-b57066450c1c; oad_n=a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222023-10-30+14%3A52%3A44%22%3B%7D; __jsluid_s=9cf863bb9c9c7a233b17e918f2228c74; __omc_chl=; __omc_r=; __mfwc=direct; uva=s%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1698648764%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B; __mfwurd=a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1698648764%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D; __mfwuuid=653f52bc-914c-8441-6d82-b57066450c1c; bottom_ad_status=0; __mfwa=1698648764837.76974.3.1698653932344.1698656825781; __mfwlv=1698656825; __mfwvn=2; PHPSESSID=nl5p6aadgj31qpm1pvckgj4a20; Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698648765,1698654537,1698656826,1698656945; __mfwb=ad354a1a267b.3.direct; __mfwlt=1698656964; Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698656965',
        'origin': 'https://www.mafengwo.cn',
        'pragma': 'no-cache',
        'referer': 'https://www.mafengwo.cn/gonglve/',
        'sec-ch-ua': '"Chromium";v="118", "Microsoft Edge";v="118", "Not=A?Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'same-origin',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46',
        'x-requested-with': 'XMLHttpRequest',
    }
    ip_list = []
    url = "https://api.hailiangip.com:8522/api/getIpEncrypt?dataType=0&encryptParam=SlDyzgfgDW12vuaMHmQkMz9pKEmWH7kDAoD1ZC4KkxpHXuvLm%2B3L9xWaasGPtq4T4c1NrsDYZz0rKXbmVK5B4QBG7L4Me9DYR7C%2FmLj0ijWqRxTDf5%2F5k7dF8PzzjCTrmjnX%2FnL660JPzNmFbcKWe0Y75OsMrICvGcnqo6oq1zpYDcvC%2FeTFVDIQgCd6Bl9z0AOtabf92lu9Nd5cdhcIj9TXdJz%2BFe7XNmfGghK3pfs%3D"

    for i in range(1, 51):
        data = {
            'page': f"{i}",
        }
        dict_data = requests.get(url).json()
        proxy = {
            "https://": str(dict_data["data"][0]["ip"]) + ":" + str(dict_data["data"][0]["port"])
        }
        response = requests.post('https://www.mafengwo.cn/gonglve/', cookies=cookies, headers=headers, data=data,
                                 proxies=proxy).text
        # print(response)
        html = etree.HTML(response)
        data_list = html.xpath("//div[@class='_j_feed_data']//div[@class='feed-item _j_feed_item']")

        for num, data in enumerate(data_list):
            list2 = []
            href = data.xpath("./a/@href")[0]
            # num=href.split("/")[-1] + ".html"
            num = f"two-{i}{num}"
            from_one = "来自" + data.xpath("./a/div[1]/span[2]/strong/text()")[0]
            title = (data.xpath("./a/div[2]/text()")[0]).strip()
            try:
                fengfeng = data.xpath("./a/div[1]/span[1]/span/text()")[0] + (
                    data.xpath("./a/div[1]/span[1]/text()")[0]).strip()
            except:
                fengfeng = "None"
            try:
                liulan = data.xpath("./a/div[4]/ul/li[4]/text()")[0]
            except:
                liulan = "None"
            try:
                pinglun = data.xpath("./a/dl/dd/div[2]/span[2]/text()")[0]
            except:
                pinglun = "None"
            list2.append(href)
            list2.append(from_one)
            list2.append(title)
            list2.append(fengfeng)
            list2.append(liulan)
            list2.append(pinglun)
            # list2.append("\n")
            # print(href, from_one, title, fengfeng, liulan, pinglun)
            xiang_data(href, proxy, num)

            # 创建csv文档对象
            with open('/document_data.csv', 'a', newline='', encoding="utf-8") as csvfile:
                writer = csv.writer(csvfile)
                # 写入数据
                writer.writerow(list2)


def xiang_data(href, proxy, num):
    headers = {
        "Cookie": "mfw_uuid=653f52bc-914c-8441-6d82-b57066450c1c; oad_n=a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222023-10-30+14%3A52%3A44%22%3B%7D; __jsluid_s=9cf863bb9c9c7a233b17e918f2228c74; __omc_chl=; __omc_r=; __mfwc=direct; uva=s%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1698648764%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B; __mfwurd=a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1698648764%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D; __mfwuuid=653f52bc-914c-8441-6d82-b57066450c1c; bottom_ad_status=0; PHPSESSID=mpms0rj465809fvpbo43tn36m3; __mfwa=1698648764837.76974.7.1698714855088.1698736585468; __mfwlv=1698736585; __mfwvn=5; Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698709704,1698714855,1698715591,1698736586; __jsluid_h=e295a25f9cad6c823cc0f83b82029ca5; __jsl_clearance=1698739353.166|0|iX%2F0UpV0%2F%2FKpDKUYaVaApf9vjhY%3D; __jsl_clearance_s=1698741026.535|0|zGz2woaszAZi%2B34xpW4cWwG0MVc%3D; __mfwb=25b7c2e7fe94.11.direct; __mfwlt=1698741029; Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0=1698741029",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.76",
        "Referer": "https://www.mafengwo.cn/gonglve/"
    }
    response = requests.get(url=href, headers=headers, proxies=proxy).text
    with open("./xqy/" + num + ".html", "w", encoding="utf-8")as file:
        file.write(response)
    print("保存成功")


get_start_data()
post_start_data()
